﻿using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Windows.Forms;
using System.Collections;
using System.IO;
using System.Net;
using System.Text.RegularExpressions;
using System.Threading;


namespace HW_7_g1
{
    


    public partial class Form1 : Form
    {
        public string startUrl = "";
        public Form1()
        {
            InitializeComponent();
            textBox1.Text = "https://www.whu.edu.cn/";
        }

        private void textBox1_TextChanged(object sender, EventArgs e)
        {
            this.startUrl = textBox1.Text;
        }

        private void button1_Click(object sender, EventArgs e)
        {
            textBox2.Text = "";
            Crawler myCrawler = new Crawler();
            myCrawler.startUrl = this.startUrl;
            myCrawler.urls.Add(startUrl, false);
            myCrawler.Crawl(textBox2);
        }
    }



    public class Crawler
    {
        public Hashtable urls = new Hashtable();
        public int count = 0;
        public string startUrl = "";

        public void Crawl(TextBox t)
        {
            Console.WriteLine("开始爬行...");
            t.Text += "开始爬行...\n";
            while (true)
            {
                string current = null;
                foreach (string url in urls.Keys)
                {
                    // 找到未爬过的url
                    if ((bool)urls[url]) continue;
                    current = url;
                }
                // 所有url都爬过或者爬过的url数量超过十个，则停止
                if (current == null || count > 50) break;

                Console.WriteLine("爬行" + current + "页面！");
                t.Text = t.Text  + current + "\n";
                // 获得某个页面的html文本信息
                string html = DownLoad(current, t);
                // 标记该url爬过了
                urls[current] = true;
                count++;
                // 解析该界面
                Parse(html);
            }
            Console.WriteLine("爬行结束");
            t.Text += "爬行结束\n";
        }

        public string DownLoad(string url, TextBox t)
        {
            try
            {
                WebClient webClient = new WebClient();
                webClient.Encoding = Encoding.UTF8;
                string html = webClient.DownloadString(url);

                string fileName = count.ToString();
                File.WriteAllText(fileName + ".html", html, Encoding.UTF8);

                return html;
            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.Message);
                t.Text += ex.Message;
                return "";
            }
        }

        public void Parse(string html)
        {
            // 只爬取当前界面下属的html文件，不解析跳转的文件
            string strRef = @"(href|HREF)[]*=[]*[""'][^(http|https)][^""'#>]+.(htm|html|shtml)[""']";
            // 看这个界面还有没有html文件可以爬
            MatchCollection matches = new Regex(strRef).Matches(html);
            foreach (Match match in matches)
            {
                // 若有，则将html的域名放入urls中，之后再爬取
                strRef = this.startUrl + match.Value.Substring(match.Value.IndexOf('=') + 1).Trim('"', '\"', '#', '>');
                if (strRef.Length == 0) continue;
                if (urls[strRef] == null) urls[strRef] = false;
            }
        }
    }
}
